/**
 * 
 */
package ca.uwindsor.cs.deepweb.estimation;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Date;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.LockObtainFailedException;

/**
 * This indexer is able to read all the files in a specified directory. Each
 * content of a file will be as the content field of a Lucene document as well
 * as the file name will be the ID field of the document. Only content field
 * will be indexed.
 * 
 * @author Liang Jie
 * 
 */
public class FilesContentIndexer {

	/**
	 * Index working path
	 */
	// public static final String PATH = System.getProperty("user.dir")
	// + java.io.File.separator + "Experiment" + java.io.File.separator
	// + "en_wiki";
	/**
	 * The name of the Field which is used to store value in Document
	 */
	public static final String FIELD_CONTENT = "CONTENT";

	/**
	 * The name of the Field which is used to identify the Document globally.
	 * Set globally unique ID if you need to query in different indexes and
	 * compare the result.
	 */
	public static final String FIELD_ID = "ID";

	/**
	 * The name of the Field which is used to store title in Document
	 */
	public static final String FIELD_TITLE = "TITLE";

	/**
	 * The name of the Field which is used to store the size of Document
	 */
	public static final String FIELD_SIZE = "SIZE";

	/**
	 * The name of the Field which is used to identify the Document locally. Set
	 * unique ID if you don't need to query in different indexes.
	 */
	public static final String FIELD_INTERNAL_ID = "INTERNAL_ID";

	/**
	 * The writer to write the index to a directory Specified by PATH
	 * 
	 * @see #indexDir
	 */
	protected IndexWriter writer;

	/**
	 * The directory which contains the files to be indexed
	 */
	protected File dataDir;

	/**
	 * The directory restore the index files
	 */
	protected String indexDir;

	/**
	 * The encoding of the file content
	 */
	protected String encoding;

	/**
	 * Whether or not display detailed information
	 */
	protected boolean isVerbose;

	/**
	 * Whether or not create the index file
	 */
	protected boolean isCreate;

	/**
	 * The extension of the files to be indexed
	 */
	protected String extension;

	/**
	 * The ID for each document;
	 */
	protected long globalID;

	/**
	 * The required extensions of files to index
	 */
	protected String[] extensions;

	// /**
	// * The Analyzer
	// */
	// protected Analyzer luceneAnalyzer;
	//	
	// /**
	// * The index writer
	// */
	// protected IndexWriter indexWriter;

	/**
	 * Constructor, initialize the object of DataFile
	 * 
	 * @param sourcepath
	 *            The directory which contains the files to be indexed
	 * @param indexdir
	 *            The directory which contains the Lucene index files
	 */
	public FilesContentIndexer(String sourcepath, String indexdir) {
		this(sourcepath, indexdir, "UTF-8");
	}

	/**
	 * Constructor, initialize the object of DataFile
	 * 
	 * @param sourcepath
	 *            The directory which contains the files to be indexed
	 * @param indexdir
	 *            The directory which contains the Lucene index files
	 * @param encoding
	 *            The encoding of reading this file
	 */
	public FilesContentIndexer(String sourcepath, String indexdir,
			String encoding) {
		this(sourcepath, indexdir, encoding, false, true, "txt");
	}

	/**
	 * Constructor, initialize the object
	 * 
	 * @param sourcepath
	 *            The directory which contains the files to be indexed
	 * @param indexdir
	 *            The directory which contains the Lucene index files
	 * @param verbose
	 *            Whether or not display detailed information
	 */
	public FilesContentIndexer(String sourcepath, String indexdir,
			boolean verbose) {
		this(sourcepath, indexdir, "UTF-8", verbose, true, "txt");
	}

	/**
	 * Constructor, initialize the object of DataFile
	 * 
	 * @param sourcepath
	 *            The directory which contains the files to be indexed
	 * @param indexdir
	 *            The directory which contains the Lucene index files
	 * @param encoding
	 *            The encoding of reading this file
	 * @param verbose
	 *            Whether or not display detailed information
	 * @param create
	 *            Whether or not create the index file
	 * @param ext
	 *            The extension of the files send to index
	 */
	public FilesContentIndexer(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create) {
		this(sourcepath, indexdir, encoding, verbose, create, "txt");
	}

	/**
	 * Constructor, initialize the object of DataFile
	 * 
	 * @param sourcepath
	 *            The directory which contains the files to be indexed
	 * @param indexdir
	 *            The directory which contains the Lucene index files
	 * @param encoding
	 *            The encoding of reading this file
	 * @param verbose
	 *            Whether or not display detailed information
	 * @param create
	 *            Whether or not create the index file
	 * @param ext
	 *            The extension of the files send to index
	 */
	public FilesContentIndexer(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create, String ext) {
		this(sourcepath, indexdir, encoding, verbose, create,
				new String[] { ext });
	}

	/**
	 * Constructor, initialize the object of DataFile
	 * 
	 * @param sourcepath
	 *            The directory which contains the files to be indexed
	 * @param indexdir
	 *            The directory which contains the Lucene index files
	 * @param encoding
	 *            The encoding of reading this file
	 * @param verbose
	 *            Whether or not display detailed information
	 * @param create
	 *            Whether or not create the index file
	 * @param ext
	 *            The extensions of the files send to index
	 */
	public FilesContentIndexer(String sourcepath, String indexdir,
			String encoding, boolean verbose, boolean create, String[] ext) {
		dataDir = new File(sourcepath);
		indexDir = indexdir;
		this.encoding = encoding;
		isVerbose = verbose;
		isCreate = create;
		extensions = ext;
		globalID = 0;
		// luceneAnalyzer = new StandardAnalyzer();
	}

	/**
	 * @param encoding
	 *            the encoding to set
	 */
	protected void setEncoding(String encoding) {
		this.encoding = encoding;
	}

	/**
	 * @return the encoding
	 */
	protected String getEncoding() {
		return encoding;
	}

	/**
	 * @param isVerbose
	 *            the isVerbose to set
	 */
	protected void setVerbose(boolean isVerbose) {
		this.isVerbose = isVerbose;
	}

	/**
	 * @return the isVerbose
	 */
	protected boolean isVerbose() {
		return isVerbose;
	}

	/**
	 * @param isCreate
	 *            the isCreate to set
	 */
	protected void setCreate(boolean isCreate) {
		this.isCreate = isCreate;
	}

	/**
	 * @return the isCreate
	 */
	protected boolean isCreate() {
		return isCreate;
	}

	/**
	 * @param globalID
	 *            the globalID to set
	 */
	protected void setStartID(long globalID) {
		this.globalID = globalID;
	}

	/**
	 * @return the globalID
	 */
	protected long getGlobalID() {
		return globalID;
	}

	/**
	 * Check whether a files is required to be indexed using the given
	 * extension(s)
	 * 
	 * @param filename
	 * @return
	 */
	protected boolean isValidExtension(String filename) {
		for (int i = 0; i < extensions.length; i++) {
			if (filename.endsWith("." + extensions[i])) {
				return true;
			}
		}
		return false;
	}

	/**
	 * Start to index the files in the specified directory. One file's content
	 * is an instance of Document in Lucene.
	 */
	public void generateIndex() {
		try {
			Analyzer luceneAnalyzer = new StandardAnalyzer();
			File[] dataFiles = dataDir.listFiles();
			IndexWriter indexWriter;
			indexWriter = new IndexWriter(indexDir, luceneAnalyzer, isCreate);

			long startTime = new Date().getTime();
			// long id = 1;

			// Add documents to the index
			for (int i = 0; i < dataFiles.length; i++) {
				if (dataFiles[i].isFile()
						&& isValidExtension(dataFiles[i].getName())) {
					if (isVerbose) {
						System.out.println("Indexing file "
								+ dataFiles[i].getCanonicalPath());
					}
					Document document = new Document();
					InputStreamReader txtReader = new InputStreamReader(
							new FileInputStream(dataFiles[i]), encoding);

					// make the ID from the path shorter - get the file name
					// String file = dataFiles[i].getCanonicalPath();
					// int offset1 = file.lastIndexOf(java.io.File.separator);
					// int offset2 = file.lastIndexOf(".");
					// String word = new String(file.substring(offset1 + 1,
					// offset2));
					//
					String word = String.valueOf(globalID);
					// System.out.println(id);

					Field FieldPath = new Field(FilesContentIndexer.FIELD_ID,
							word, Field.Store.YES, Field.Index.UN_TOKENIZED);
					Field FieldBody = new Field(
							FilesContentIndexer.FIELD_CONTENT, txtReader);

					document.add(FieldPath);
					document.add(FieldBody);
					indexWriter.addDocument(document);
					txtReader.close();
					globalID++;
				}
			}
			long endTime = new Date().getTime();
			System.out
					.println("It takes "
							+ (endTime - startTime)
							+ " milliseconds to create index for the files in directory "
							+ dataDir.getPath()
							+ "\nTotal documnents indexed is:" + (globalID - 1));

			System.out
					.println("index is completed! \n Start to optimize the index...");
			indexWriter.optimize();
			System.out.println("optimization is completed!");
			indexWriter.close();

		} catch (CorruptIndexException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (LockObtainFailedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
			System.exit(1);
		}

	}

}
